#!/usr/bin/env python3

import numpy as np

def calc_direct(data):
	x, t = zip(*data)
	S1 = sum([_**2 for _ in x])
	S2 = sum(x)
	M1 = np.array([[S1, S2], [S2, len(data)]])
	M2 = np.array([x, [1]*len(data)])
	WB = np.linalg.inv(M1).dot(M2).dot(t)
	return WB

class SGD:
	def __init__(self, lr = 0.01):
		self.lr = lr

	def update(self, w, b, dw, db):
		w = w - self.lr * dw
		b = b - self.lr * db
		return w, b

class Momentum:
	def __init__(self, lr = 0.01, momentum = 0.9):
		self.lr = lr
		self.momentum = momentum
		self.vw = None
		self.vb = None

	def update(self, w, b, dw, db):
		if self.vw is None:
			self.vw = np.zeros_like(w)
			self.vb = np.zeros_like(b)

		self.vw = self.momentum * self.vw - self.lr * dw
		self.vb = self.momentum * self.vb - self.lr * db
		w = w + self.vw
		b = b + self.vb
		return w, b

class AdaGrad:
	def __init__(self, lr = 0.01):
		self.lr = lr
		self.hw = None
		self.hb = None

	def update(self, w, b, dw, db):
		if self.hw is None:
			self.hw = np.zeros_like(w)
			self.hb = np.zeros_like(b)

		self.hw = self.hw + dw * dw
		self.hb = self.hb + db * db
		w = w - self.lr * dw / (np.sqrt(self.hw) + 1e-7)
		b = b - self.lr * db / (np.sqrt(self.hb) + 1e-7)
		return w, b

def bp_1by1(w, b, data):
	opt = SGD()
	
	for (x, t) in data:
		y = w*x + b
		L = (y - t)**2/2.
		dw, db = (y-t)*x, (y-t)
		w, b = opt.update(w, b, dw, db)
		yield w, b, L, x, t, y, dw, db

training = [(0, 1), (1, 2.9), (2, 5.1)]

for i in range(3, 20):
	delta = np.random.normal(0, 0.2, 1)[0]
	training.append((i, 2*i + 1 + delta))

np.random.shuffle(training)

WB = calc_direct(training)
print(f'W: {WB[0]:.04f}, B: {WB[1]:.04f}')

w, b = 0.5, 0.6
for epoch in range(3):
	np.random.shuffle(training)
	print('='*40)
	for (w, b, L, x, t, y, dw, db) in bp_1by1(w, b, training):
		print(f'w: {w:.04f}, b: {b:.04f}, L: {L:.04f}, x: {x:.04f}, t: {t:.04f}, y: {y:0.4f}, dw: {dw:0.4f}, db: {db:.04f}')
